package cs.dm.ics.data;

/**
 * This class pre-processes the data file, cleans them and rewrites them into a new file
 * @author Nan,Eyrun
 *
 */
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Scanner;
//import java.util.Set;
import java.text.SimpleDateFormat;
import java.text.ParseException;
import java.util.Date;
import java.io.*;

//import sun.misc.Cleaner;

public class DataCleansing {

	//to store the cleaned data
	private ArrayList<String[]> cleansedData = new ArrayList<String[]>();
	
	//to store the mapping between item id and the list of item names (Chinese)
	private Hashtable<String, ArrayList<String>> itemIDNameMapping = new Hashtable<String, ArrayList<String>>();
	//to store the mapping between item name (Chinese) and the list of item ids
	private Hashtable<String, ArrayList<String>> itemNameIDMapping = new Hashtable<String, ArrayList<String>>();
	//to store the <date, customer id> key and the basket(the list of items purchased by the customer on that date)
	private Hashtable<String, ArrayList<String>> baskets = new Hashtable<String, ArrayList<String>>();
	
	//this field records down the # of transactions that have a negative number as the quantity of the items purchased
	private int returnTransactionNum = 0;
	//this filed records downt the # of transactions that have 0 as the quantity of the items purchased
	private int emptyTransactionNum = 0;
	//this filed records downt the # of transactions that have a value that is >1 as the quantity of the items purchased
	private int multiplePurchaseTransactionNum = 0;
	//total number of transactions
	private int totalTransactionsNum = 0;
	
	//input file name
	//private final String rawDataFile = "HYK_JL_V1.TXT";
	private final String rawDataFile = "test";
	//output file names
	private final String cleansedDataFile = "cleansedTransactions";
	private final String invalidDataFile = "invalidTransactions"; //transactions whose quantity of items is not 1 and transactions whose item name contains space in it
	private final String statisticsFile = "transactionStatistics"; //statistics gained over the cleansing
	private final String idNameMappingFile = "itemIdNameMapping"; //mapping between item id and list of names
	private final String nameIdMappingFile = "itemNameIdMapping"; //mapping between name and list of ids
	private final String basketFile = "baskets"; //date, customer id, list of items he purchased
	
	//read/write
	public Hashtable<String, ArrayList<String>> getBaskets(){
		return this.baskets;
	}
	
	//data cleaning process
	public void cleaningData(){
		try{
			BufferedReader br = new BufferedReader(new FileReader(rawDataFile)); 
			PrintWriter invalidDataFileWriter = new PrintWriter(new FileWriter(invalidDataFile, true));
			//OutputStreamWriter invalidDataFileWriter = new OutputStreamWriter(new FileOutputStream(invalidDataFile, true),"GB2312"); //in order to write chinese characters
			PrintWriter cleansedDataFileWriter = new PrintWriter(new FileWriter(cleansedDataFile, true));
			PrintWriter statisticsFileWriter = new PrintWriter(new FileWriter(statisticsFile, true));
			PrintWriter idNameMappingFileWriter = new PrintWriter(new FileWriter(idNameMappingFile, true));
			//OutputStreamWriter idNameMappingFileWriter = new OutputStreamWriter(new FileOutputStream(idNameMappingFile, true),"GB2312"); 
			PrintWriter nameIdMappingFileWriter = new PrintWriter(new FileWriter(nameIdMappingFile, true));
			//OutputStreamWriter nameIdMappingFileWriter = new OutputStreamWriter(new FileOutputStream(nameIdMappingFile, true),"GB2312"); 
			PrintWriter basketFileWriter = new PrintWriter(new FileWriter(basketFile, true));
			
			String temp = "";
			Scanner sc;
			String[] tokens = new String[12]; //tokens of each line from the raw data file
			SimpleDateFormat inputFormat = new SimpleDateFormat("MMMddyyyy");
			SimpleDateFormat outputFormat = new SimpleDateFormat("yyyyMMdd");
			Date transDate;
			
			//this is one line in the cleaned data output formatted as:
			//transaction id, date, unit value, unit discount, item id, customer id
			String[] cleansedTransaction;
			//the date read from the raw data
			String dateString = "";
			
			while((temp = br.readLine())!= null){
				//counting the number of transactions in the raw data file
				totalTransactionsNum ++;
				System.out.println(totalTransactionsNum + "****");
				cleansedTransaction = new String[6];
				
				sc = new Scanner(temp).useDelimiter("\\s+");
				
				//read all the tokens into the array
				int i = 0;
				boolean flag = false;
				while(sc.hasNext()){
					//System.out.println(i);
					if(i >= 12){//there is space in the item's name
						flag = true;
						break;
					}
					tokens[i] = sc.next();
					i ++;
				}
				if(flag == true){//if in this record, the item name contains space
					invalidDataFileWriter.write(temp+"\n");
					continue;
				}
				if(i < 12){//there are less than 12 tokens in this line 
					invalidDataFileWriter.write(temp+"\n");
					continue;
				}
				
				/**the following is to assign the values from the raw data to the new cleansed data**/
				//quantity of item
				if(Double.parseDouble(tokens[6]) == 0){
					emptyTransactionNum ++;
					invalidDataFileWriter.write(temp+"\n");
					continue;
				}
				else if (Double.parseDouble(tokens[6]) < 0)
				{
					returnTransactionNum ++;
					invalidDataFileWriter.write(temp+"\n");
					continue;
				}
				else if (Double.parseDouble(tokens[6]) > 1)
				{
					multiplePurchaseTransactionNum ++;
					invalidDataFileWriter.write(temp+"\n");
				}				
				
				//transaction id
				cleansedTransaction[0] = tokens[1];
				
				//date
				if(Integer.parseInt(tokens[3]) < 10){//the day of the date has only 1 digit
					tokens[3] = "0" + tokens[3];
				}
				dateString = tokens[2]+tokens[3]+tokens[4];
				transDate = inputFormat.parse(dateString);
				cleansedTransaction[1] = outputFormat.format(transDate);
				if(cleansedTransaction[1].indexOf("0007") == 0){
					System.err.println("Wrong!!"+totalTransactionsNum);
				}
				
				//unit value
				cleansedTransaction[2] = Double.toString((Double.parseDouble(tokens[7]) / Double.parseDouble(tokens[6])));
				
				//unit discount
				cleansedTransaction[3] = Double.toString((Double.parseDouble(tokens[8]) / Double.parseDouble(tokens[6])));
				
				
				//write a "product name to item id" mapping into a hash table
				if (!itemNameIDMapping.containsKey(tokens[10]))
				{
					ArrayList<String> collidedIds = new ArrayList<String>();
					collidedIds.add(tokens[9]);
					itemNameIDMapping.put(tokens[10], collidedIds);
				}
				else
				{
					itemNameIDMapping.get(tokens[10]).add(tokens[9]);
					// There is a collision of item ids for the same item name so we ignore this record and go to the next one
					continue;
				}
				
				//write a "item id to product name" mapping into a hash table
				if (!itemIDNameMapping.containsKey(tokens[9]))
				{
					ArrayList<String> collidedNames = new ArrayList<String>();
					collidedNames.add(tokens[10]);
					itemIDNameMapping.put(tokens[9], collidedNames);
				}
				else
				{
					itemIDNameMapping.get(tokens[10]).add(tokens[9]);
					// There is a collision of item names for the same item id so we ignore this record and go to the next one
					continue;
				}
				
				
				//item id
				cleansedTransaction[4] = tokens[9];
				
				//customer id
				cleansedTransaction[5] = tokens[11];
				
				cleansedData.add(cleansedTransaction);
				
				//clustering the cleansed data into baskets
				String basketKey;
				basketKey = cleansedTransaction[1] + "-" + cleansedTransaction[5];
				if (!baskets.containsKey(basketKey))
				{
					ArrayList<String> items = new ArrayList<String>();
					items.add(cleansedTransaction[4]);
					baskets.put(basketKey, items);
				}
				else
				{
					baskets.get(basketKey).add(cleansedTransaction[4]);
				}
			}
			
			//To write the cleansed data to a file
			for (String[] transaction : cleansedData)
			{
				for (int i =0; i<6; i++)
				{
					if (i==5)
						cleansedDataFileWriter.write(transaction[i]);
					else
						cleansedDataFileWriter.write(transaction[i] + ",");
				}
				cleansedDataFileWriter.write("\n");
			}
			
			//To write the statistics data to a file
			statisticsFileWriter.write("Percentage of returned transactions: " + (double)returnTransactionNum/(double)totalTransactionsNum);
			statisticsFileWriter.write("Percentage of empty transactions: " + (double)emptyTransactionNum/(double)totalTransactionsNum);
			statisticsFileWriter.write("Percentage of multiple item transactions: " + (double)multiplePurchaseTransactionNum/(double)totalTransactionsNum);
			
			//To write the "item id to item name" mapping into a file
			Iterable<String> itemIds = itemIDNameMapping.keySet();
			Iterator<String> it = itemIds.iterator();
			while(it.hasNext())
			{
				String id = it.next();
				idNameMappingFileWriter.write(id);
				ArrayList<String> names = itemIDNameMapping.get(id);
				for (String s : names)
				{
					idNameMappingFileWriter.write("," + s);
				}
				idNameMappingFileWriter.write("\n");
			}
			
			//To write the "item name to item id" mapping into a file
			Iterable<String> itemNames = itemNameIDMapping.keySet();
			Iterator<String> it2 = itemNames.iterator();
			while(it2.hasNext())
			{
				String name = it2.next();
				nameIdMappingFileWriter.write(name);
				ArrayList<String> ids = itemNameIDMapping.get(name);
				for (String id : ids)
				{
					nameIdMappingFileWriter.write("," + id);
				}
				nameIdMappingFileWriter.write("\n");
			}
			
			//To write the baskets into a file
			Iterable<String> basketKeys = baskets.keySet();
			Iterator<String> it3 = basketKeys.iterator();
			while(it3.hasNext())
			{
				String basketKey = it3.next();
				basketFileWriter.write(basketKey);
				ArrayList<String> items = baskets.get(basketKey);
				for (String itemId : items)
				{
					basketFileWriter.write("," + itemId);
				}
				basketFileWriter.write("\n");
			}
			
			cleansedDataFileWriter.flush();
			statisticsFileWriter.flush();
			invalidDataFileWriter.flush();
			idNameMappingFileWriter.flush();
			nameIdMappingFileWriter.flush();
			basketFileWriter.flush();
			
			cleansedDataFileWriter.close();
			statisticsFileWriter.close();
			invalidDataFileWriter.close();
			idNameMappingFileWriter.close();
			nameIdMappingFileWriter.close();
			basketFileWriter.close();
			
			//to release memory
			cleansedData = null;
			itemIDNameMapping = null;
			itemNameIDMapping = null;
			baskets = null;
			
		}
		catch(IOException ioe){
			ioe.printStackTrace();
		}
		catch(ParseException pe){
			pe.printStackTrace();
		}
	}
	
	public static void main(String[] args){
		DataCleansing cd = new DataCleansing();
		cd.cleaningData();
	}
}
